{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## 1.3.4 计算图"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import numpy as np"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:16:01.640001100Z",
     "start_time": "2023-05-03T14:16:01.533999300Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.3.4.3 Repeat节点"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:16:40.390017300Z",
     "start_time": "2023-05-03T14:16:40.379015100Z"
    }
   },
   "outputs": [],
   "source": [
    "D, N = 8, 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "x = np.random.randn(1, D)  # 输入"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:17:01.732310200Z",
     "start_time": "2023-05-03T14:17:01.723310600Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "y = np.repeat(x, N, axis=0)  # 正向传播"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:17:36.949257200Z",
     "start_time": "2023-05-03T14:17:36.931260500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293],\n       [-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293],\n       [-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293],\n       [-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293],\n       [-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293],\n       [-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293],\n       [-0.17808531,  1.39669766, -0.70718715,  1.98144475,  0.88642414,\n        -0.52805048, -1.53257264, -0.71737293]])"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:17:41.494685100Z",
     "start_time": "2023-05-03T14:17:41.481684500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "dy = np.random.randn(N, D)  # 假设的梯度"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:18:16.560267700Z",
     "start_time": "2023-05-03T14:18:16.530268Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "dx = np.sum(dy, axis=0, keepdims=True)  # 反向传播"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:18:48.315498500Z",
     "start_time": "2023-05-03T14:18:48.306496700Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[ 2.37796747, -1.63240079,  1.41024062,  0.21443381, -2.24055111,\n         1.61517225,  0.2525587 , -2.86137176]])"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:18:52.152959200Z",
     "start_time": "2023-05-03T14:18:52.148960900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "这里通过 np.repeat() 方法进行元素的复制。上面的例子中将复制 N 次\n",
    "数组 x。通过指定 axis，可以指定沿哪个轴复制。因为反向传播时要计算\n",
    "总和，所以使用 NumPy 的 sum() 方法。此时，通过指定 axis 来指定对哪\n",
    "个轴求和。另外，通过指定 keepdims=True，可以维持二维数组的维数。在\n",
    "上面的例子中，当 keepdims=True 时，np.sum() 的结果的形状是 (1, D)；当\n",
    "keepdims=False 时，形状是 (D,)。\n",
    "\"\"\""
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.3.4.4 Sum节点"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "D, N = 8, 7\n",
    "x = np.random.randn(N, D)  # 输入"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:20:26.671461Z",
     "start_time": "2023-05-03T14:20:26.655460400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "y = np.sum(x, axis=0, keepdims=True)  # 正向传播"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:20:27.489595300Z",
     "start_time": "2023-05-03T14:20:27.473597100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "dy = np.random.randn(1, D)  # 假设的梯度\n",
    "dx = np.repeat(dy, N, axis=0)  # 反向传播"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:20:27.861705200Z",
     "start_time": "2023-05-03T14:20:27.848705900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[ 1.65196777, -1.32061663, -1.13073388, -1.67569885,  3.15396898,\n        -0.3089492 ,  1.2983359 ,  3.54507303]])"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:20:35.000302800Z",
     "start_time": "2023-05-03T14:20:34.988303400Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "array([[-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219],\n       [-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219],\n       [-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219],\n       [-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219],\n       [-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219],\n       [-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219],\n       [-1.58780172,  1.75302864, -0.31538166, -1.62901763, -2.20836159,\n        -1.97943957, -0.51309289,  0.04613219]])"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:20:37.725511Z",
     "start_time": "2023-05-03T14:20:37.712511200Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "class MatMul:\n",
    "    def __init__(self, W):\n",
    "        self.params = [W]\n",
    "        self.grads = [np.zeros_like(W)]\n",
    "        self.x = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        W, = self.params\n",
    "        out = np.dot(x, W)\n",
    "        self.x = x\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        W, = self.params\n",
    "        dx = np.dot(dout, W.T)\n",
    "        dW = np.dot(self.x.T, dout)\n",
    "        self.grads[0][...] = dW\n",
    "        return dx"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:40:19.966918500Z",
     "start_time": "2023-05-03T14:40:19.939863400Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1.3.5 梯度的推导和反向传播的实现\n",
    "### 1.3.5.1 Sigmoid 层"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "class Sigmoid:\n",
    "    def __init__(self):\n",
    "        self.params, self.grads = [], []\n",
    "        self.out = None\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = 1 / (1 + np.exp(-x))\n",
    "        self.out = out\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = dout * (1.0 - self.out) * self.out\n",
    "        return dx"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T14:45:21.724436Z",
     "start_time": "2023-05-03T14:45:21.717433100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "class SGD:\n",
    "    def __init__(self, lr=0.01):\n",
    "        self.lr = lr\n",
    "\n",
    "    def update(self, params, grads):\n",
    "        for i in range(len(params)):\n",
    "            params[i] -= self.lr * grads[i]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T15:24:55.169985200Z",
     "start_time": "2023-05-03T15:24:55.152985100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('..')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T15:25:43.046895500Z",
     "start_time": "2023-05-03T15:25:43.034894Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "from dataset import spiral\n",
    "import matplotlib.pyplot as plt"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T15:26:22.905595700Z",
     "start_time": "2023-05-03T15:26:22.162172Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x (300, 2)\n",
      "t (300, 3)\n"
     ]
    }
   ],
   "source": [
    "x, t = spiral.load_data()\n",
    "print('x', x.shape) # (300, 2)\n",
    "print('t', t.shape) # (300, 3)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-05-03T15:26:34.850400700Z",
     "start_time": "2023-05-03T15:26:34.821399500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
